{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f1029e76-7e61-4296-801d-3a49637b5972",
   "metadata": {},
   "source": [
    "# LLM-as-a-Judge Evaluation\n",
    "\n",
    "Evaluating model output with an LLM.\n",
    "\n",
    "Add `prettytable` to make a nice looking output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64f96a0a-bb6e-4dbf-a37a-1f602429fb47",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install prettytable==3.10.2 openai==1.39.0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d27c7499-d8ca-4fa3-8a29-d675bde7bafc",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 show prettytable openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7175e70a-abf6-4c49-9a04-4abec23c646f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipython_secrets\n",
    "import os\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] = ipython_secrets.get_secret('OPENAI_API_KEY')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3650ce4e-c63a-4f96-968f-bffbfd030d3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from prettytable import PrettyTable\n",
    "import openai\n",
    "import os\n",
    "\n",
    "# Add your OpenAI API key to call the model\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "# Data to evaluate\n",
    "evaluation_data = [\n",
    "    {\n",
    "        \"input\": \"What should I do in New York City in July?\",\n",
    "        \"output\": \"Check out Times Square, go to an outdoor concert, and visit the Statue of Liberty.\",\n",
    "    },\n",
    "    {\n",
    "        \"input\": \"Can you help me with my math homework?\",\n",
    "        \"output\": \"I'm designed to assist with travel queries. For math help, try using online resources like Khan Academy or Mathway.\",\n",
    "    },\n",
    "    {\n",
    "        \"input\": \"What’s the capital of France?\",\n",
    "        \"output\": \"The capital of France is Paris.\",\n",
    "    }\n",
    "]\n",
    "\n",
    "\n",
    "# LLM-as-a-Judge Evaluation metric\n",
    "# that assesses if the output includes a recommendation.\n",
    "def evaluate_includes_recommendation(input, output):\n",
    "    # Few-shot examples to help the model produce better answers.\n",
    "    few_shot_examples = [\n",
    "        {\n",
    "            \"input\": \"What are some good restaurants in Paris?\",\n",
    "            \"output\": \"Try Le Jules Verne for an upscale dining experience, or visit Le Relais de l'Entrecôte for a classic steak frites.\",\n",
    "            \"recommendation\": True\n",
    "        },\n",
    "        {\n",
    "            \"input\": \"Where should I stay in London?\",\n",
    "            \"output\": \"Consider staying at The Ritz for luxury or the Hoxton for a more budget-friendly option.\",\n",
    "            \"recommendation\": True\n",
    "        },\n",
    "        {\n",
    "            \"input\": \"What’s the weather like in Tokyo in winter?\",\n",
    "            \"output\": \"In winter, Tokyo is generally cool with temperatures ranging from 2°C to 12°C. While you're there, consider visiting the hot springs (onsen) for a warm and relaxing experience.\",\n",
    "            \"recommendation\": True\n",
    "        },\n",
    "        {\n",
    "            \"input\": \"What’s the population of Berlin?\",\n",
    "            \"output\": \"The population of Berlin is approximately 3.6 million.\",\n",
    "            \"recommendation\": False\n",
    "        },\n",
    "        {\n",
    "            \"input\": \"What’s the currency used in Japan?\",\n",
    "            \"output\": \"The currency used in Japan is the Japanese Yen (JPY).\",\n",
    "            \"recommendation\": False\n",
    "        }\n",
    "    ]\n",
    "\n",
    "    # Constructing the prompt\n",
    "    prompt = \"\"\"Determine whether the following output includes a recommendation based on the input.\n",
    "Format response as a JSON object with the shape { \"recommendation\": boolean }.\n",
    "Examples:\n",
    "\"\"\"\n",
    "    # Append few-shot examples to the prompt.\n",
    "    for example in few_shot_examples:\n",
    "        prompt += f\"\"\"Input: {example['input']}\n",
    "Output: {example['output']}\n",
    "Recommendation: {{ \"recommendation\": {str(example['recommendation']).lower()} }}\n",
    "\"\"\"\n",
    "\n",
    "    prompt += f\"\"\"Input: {input}\n",
    "Output: {output}\n",
    "Recommendation:\"\"\"\n",
    "\n",
    "    # Call the OpenAI API\n",
    "    response = openai.chat.completions.create(\n",
    "        # Use strong evaluator LLM\n",
    "        model=\"gpt-4o\",\n",
    "        ## Format response as JSON, so it is easier to parse\n",
    "        response_format={ \"type\": \"json_object\" },\n",
    "        messages=[{ \"role\": \"user\", \"content\": prompt }],\n",
    "        # Make sure temperature=0 for consistent outputs\n",
    "        temperature=0\n",
    "    )\n",
    "\n",
    "    recommendation = json.loads(response.choices[0].message.content)[\"recommendation\"]\n",
    "    return 1 if recommendation == True else 0\n",
    "\n",
    "def calculate_average(scores):\n",
    "    return sum(scores) / len(scores)\n",
    "\n",
    "# truncate strings for easier printing in table\n",
    "def truncate_string(s, max_length=30):\n",
    "    return (s[:max_length] + '...') if len(s) > max_length else s\n",
    "\n",
    "def create_table(data):\n",
    "    table = PrettyTable()\n",
    "    table.field_names = [\"Input\", \"Output\", \"Score\"]\n",
    "    \n",
    "    scores = [evaluate_includes_recommendation(case[\"input\"], case[\"output\"]) for case in data]\n",
    "    for case, score in zip(data, scores):\n",
    "        table.add_row([\n",
    "            truncate_string(case[\"input\"]),\n",
    "            truncate_string(case[\"output\"]),\n",
    "            score])\n",
    "    \n",
    "    # Add a blank row for visual separation\n",
    "    table.add_row([\"\", \"\", \"\"])\n",
    "    \n",
    "    # Add the average score to bottom of the table\n",
    "    average = calculate_average(scores)\n",
    "    \n",
    "    table.add_row([\"Average\", \"\", f\"{average:.4f}\"])\n",
    "    \n",
    "    return table\n",
    "\n",
    "# Create and print the table\n",
    "result_table = create_table(evaluation_data)\n",
    "print(result_table)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
